# -*- coding: utf-8 -*-
''' @author: Dinh Manh Dau <dinhmanhdau@gmail.com>
'''
import pycommonlib
import workerpool
import traceback
import datetime, time
import re, os

from pymongo import Connection
from termcolor  import cprint

LOCAL_PATH          = '/home/hoangnamhai/HarvestedData/tintuc/dantri'
MONGO_SERVER        = 'beta.mana.vn'   
MONGO_PORT          = 27017
DATABASE            = 'tintuc_v2'
PREFIX              = '/uploads/dantri' 
SITE_URL            = 'http://m.dantri.com.vn'
MAX_COUNT           = 20
MAX_ARTICLE         = 90
CONNECT             = Connection(MONGO_SERVER, MONGO_PORT)
DB                  = CONNECT[DATABASE]
ARTICLE_COLLECTION  = DB['article']
CATEGORY_COLLECTION = DB['category']
USER_COLLECTION     = DB['backend_user']
os.umask(0000)

CATEGORIES = { 'c20':     {'link' : 'xahoi',      'category': unicode('Văn hóa - Xã hội', 'utf-8'),         'tag': ['xa hoi']},    
               'c36':     {'link' : 'thegioi',    'category': unicode('Thế giới', 'utf-8'),       'tag': ['the gioi', 'quoc te']},       
               'c26':     {'link' : 'thethao',    'category': unicode('Thể thao', 'utf-8'),       'tag': ['the thao']},            
               'c23':     {'link' : 'giaitri',    'category': unicode('Giải trí', 'utf-8'),       'tag': ['giai tri']},  
               'c7':      {'link' : 'suckhoe',    'category': unicode('Sức khỏe - Giới tính', 'utf-8'),       'tag': ['suc khoe']}, 
               'c119':    {'link' : 'cong-nghe',  'category': unicode('Công nghệ số', 'utf-8'),   'tag': ['cong nghe so']},
               'c76':     {'link' : 'kinhdoanh',  'category': unicode('Kinh doanh', 'utf-8'),     'tag': ['kinh doanh']}, 
               'c111':    {'link' : 'otoxemay',   'category': unicode('Tiêu dùng', 'utf-8'),  'tag': ['oto xe may']}, 
               'c132':    {'link' : 'chuyenla',   'category': unicode('Chuyện lạ', 'utf-8'),      'tag': ['chuyen la']},
               'c135':    {'link' : 'nhipsongtre',        'category': unicode('Văn hóa - Xã hội', 'utf-8'),  'tag': ['nhip song tre']},  
               'c130':    {'link' : 'tinhyeu-gioitinh',   'category': unicode('Sức khỏe - Giới tính', 'utf-8'),  'tag': ['tinh yeu gioi tinh']},  
               'c25':     {'link' : 'giaoduc-khuyenhoc',  'category': unicode('Giáo dục', 'utf-8'),   'tag': ['giao duc']},
             }

def getRootLftCategory(name):
    try:
        if name==None or name=='': return None, None 
        root    =   CATEGORY_COLLECTION.find_one({'data': 'Tin tức'}, {'root_id': 1})
        if root==None: 
            result  =   CATEGORY_COLLECTION.find_one({'data': name}, {'root_id': 1, 'lft': 1})
        else:  
            result  =   CATEGORY_COLLECTION.find_one({'data': name, 'root_id': root['root_id']}, {'root_id': 1, 'lft': 1})
        if result==None: cprint('Category chưa tồn tại !', 'red'); return None, None
        else: return result['root_id'], result['lft']
    except:
        traceback.print_exc()

def getInfoPage(url):
    try:
        if url==None or url=='': return None, None
        maxPage     =   1000; subPath = ''
        tree        =   pycommonlib.getXMLTree(url, userAgent=True)
        pageNode    =   tree.xpath('//form/font[2]/table[4]//a')
        if len(pageNode) > 1:
            href    =   pageNode[len(pageNode) -1].get('href')
            m       =   pycommonlib.regexString('(.+)-(\d+).htm', href)
            if m:   subPath = m.group(1); maxPage = int(float(m.group(2)))
        return maxPage, subPath 
    except:
        traceback.print_exc()

def getDatetimeWithUrl(url):
   try:
       lurl     =   url.replace('m.dantri.com.vn', 'dantri.com.vn')
       tree     =   pycommonlib.getXMLTree(lurl)
       dtNode   =   tree.xpath('//div[@id="ctl00_IDContent_ctl00_divContent"]/div/span')
       if len(dtNode) > 0:
           text     =   pycommonlib.getStringWithNode(dtNode[0])
           m        =   pycommonlib.regexString('(\d+)/(\d+)/(\d+) - (\d+):(\d+)', text)
           if m:
               year = int(float(m.group(3))); month  = int(float(m.group(2))); day = int(float(m.group(1)))
               hour = int(float(m.group(4))); minute = int(float(m.group(5))) 
               return datetime.datetime(year, month, day, hour, minute) + datetime.timedelta(seconds=time.timezone)
       return datetime.datetime.utcnow()
   except:
       traceback.print_exc()

def checkArticleDuplicate(link):
    ''' Kiểm tra trùng tin tức trong DB, trả vê 1 nếu trùng, 0 trong trường hợp ngược lại
    ''' 
    try:
        if link==None or link=='': return None, None
        m           =   pycommonlib.regexString('/(\w+)-(\d+)/', link)
        if m: aId   =   '{0}-{1}'.format(m.group(1), m.group(2)) 
        else: aId   =   pycommonlib.getMd5(link)   
        result      =   ARTICLE_COLLECTION.find_one({'hashUrl' : aId})
        if result!=None:  cprint('Tin tức đã tồn tại trong cơ sở dữ liệu', 'red'); return 1, aId
        return 0, aId 
    except:
        traceback.print_exc()
        return None, None

def getAuthor(name='crawler'):
    try:
        result = USER_COLLECTION.find_one({'username': name}, {})
        if result==None: result = USER_COLLECTION.find_one({'username': 'Hindua88'}, {})
        if result!=None:
            return result['_id']
    except:
        traceback.print_exc()

def processNode(node=None):
    try:
        if node==None: return
        data = []; thumbnail = ''; flgImg = False; flgCaption = False; source_img = ''
        listNode    =   node.xpath('./*')
        for node in listNode:
            if node.tag=='img':
                result, source, img_name, size = pycommonlib.saveImage(node.get('src'), PREFIX, LOCAL_PATH)
                data.append({'data': source, 'type': 'image', 'caption': ''})
                cprint(source, 'yellow'); print 'Caption :'
            imgNode     =   node.xpath('.//img')
            if len(imgNode) > 0:
                if len(imgNode)==1:
                    text    =   pycommonlib.getStringWithNode(node)
                    result, source, img_name, size = pycommonlib.saveImage(imgNode[0].get('src'), PREFIX, LOCAL_PATH)
                    if result!=None:
                        if flgImg==False: thumbnail = source; flgImg = True
                        if text==None or len(text)<2: flgCaption = True; source_img = source; continue
                        if len(text) < 120:
                            data.append({'data': source, 'type': 'image', 'caption': text})
                            cprint('Source image : ' +  source, 'yellow'); print 'Caption :', text
                        else:
                            data.append({'data': source, 'type': 'image', 'caption': ''})
                            cprint('Source image : ' +  source, 'yellow'); print 'Caption :'
                            data.append({'data': text, 'type': 'text'})
                            cprint(text, 'green')
                else:
                    ctext = pycommonlib.getStringWithNode(node)
                    if ctext==None or ctext=='':
                        for img in imgNode:
                            result, source, img_name, size = pycommonlib.saveImage(img.get('src'), PREFIX, LOCAL_PATH)
                            data.append({'data': source, 'type': 'image', 'caption': ''})
                            cprint('Source image : ' +  source, 'yellow'); print 'Caption :'
                    else:
                        data1, thumb = processNode(node); flagError = 'error'
                        if thumbnail!='': thumbnail = thumb
                        for item in data1: data.append(item)
            else:
                content     =   pycommonlib.getStringWithNode(node)
                if flgCaption: 
                    flgCaption = False;
                    if content!=None and content!='':
                        if len(content) < 120: 
                            data.append({'data': source_img, 'type': 'image', 'caption': content})
                            cprint('Source image : ' +  source, 'yellow'); print 'Caption :', content
                        else:
                            data.append({'data': source_img, 'type': 'image', 'caption': ''})
                            cprint('Source image : ' +  source, 'yellow'); print 'Caption :'
                            data.append({'data': content, 'type': 'text'}); cprint(content, 'green')
                    else: 
                        data.append({'data': source_img, 'type': 'image', 'caption': ''})
                        cprint('Source image : ' +  source, 'yellow'); print 'Caption :' 
                    continue 
                if content!=None or content!='':
                    textBold = False; descendants = node.xpath('.//*')
                    if len(descendants) > 0:
                        for des in descendants: 
                            style = des.get('style')
                            if style and len(content) < 120:
                                if pycommonlib.regexString('bold', style): textBold = True
                    if len(content) > 1:
                        if textBold: data.append({'data': content, 'type': 'textbold'}); cprint(content, 'yellow')  
                        else: data.append({'data': content, 'type': 'text'}); cprint(content, 'green')
        return data, thumbnail  
    except:
        traceback.print_exc()

def processArticle(link, root_id, lft, tag):
    ''' Hàm xử lý chi tiết một tin tức, nếu tin đã tồn tại trong DB thì trả về: 1, ngược lại: 0
    '''
    try:
        if link==None or link=='': return
        check_exists, aId    =   checkArticleDuplicate(link)
        if check_exists==1: return 1
        print ('----------------------------------------------------------------------')
        cprint('Process article: ' + link, 'yellow')
        postedDate         =   getDatetimeWithUrl(link)
        title = thumbnail = description = ''; data = []; flgImg = False; flagError = 'success'
        flgCaption = False; source_img = '' 
        tree        =   pycommonlib.getXMLTree(link, userAgent=True)
        titleNode   =   tree.xpath('//form/font[2]/font/font')
        if len(titleNode) > 0:  title = pycommonlib.getStringWithNode(titleNode[0])
        introNode   =   tree.xpath('//form/font[2]/font/span')
        if len(introNode) > 0:
            aNode   =   introNode[0].xpath('.//a')
            if len(aNode) > 0: 
                for an in aNode: an.getparent().remove(an)
            description = pycommonlib.getStringWithNode(introNode[0])
        checkNode      =   tree.xpath('//span[@class="Content"]/*')
        if len(checkNode) > 0:
            wrapNode        =   tree.xpath('//span[@class="Content"]')
            if wrapNode[0].tag != 'span':
                data1, thumb    =   processNode(wrapNode[0])
                for item in data1: data.append(item)
                if thumb!='': thumbnail = thumb; flgImg = True
        specialNode    =   tree.xpath('//span[@class="Content"]/span')
        if len(specialNode) > 0 and specialNode[0].text:
            scontent   =   ' '.join(specialNode[0].text.split())
            if len(scontent) > 1:   data.append({'data': scontent, 'type': 'text'}); print scontent
        # remove all tag style and script
        listStyle   =   tree.xpath('//span[@class="Content"]//style')
        if len(listStyle) > 0:
            for style in listStyle: style.getparent().remove(style)
        listScript  = tree.xpath('//span[@class="Content"]//script')
        if len(listScript) > 0:
            for script in listScript: script.getparent().remove(script)
        listNode    =   tree.xpath('//span[@class="Content"]/span/*')
        if len(listNode) > 0:
            for node in listNode:
                if node.tag in ['style', 'script']: continue
                imgNode     =   node.xpath('.//img')
                if len(imgNode) > 0:
                    if len(imgNode)==1:
                        text    =   pycommonlib.getStringWithNode(node)
                        result, source, img_name, size = pycommonlib.saveImage(imgNode[0].get('src'), PREFIX, LOCAL_PATH)
                        if result!=None:
                            if flgImg==False: thumbnail = source; flgImg = True
                            if text==None or len(text)<2: flgCaption = True; source_img = source; continue
                            if len(text) < 180:
                                data.append({'data': source, 'type': 'image', 'caption': text})
                                cprint('Source image : ' +  source, 'yellow'); print 'Caption :', text
                            else:
                                data.append({'data': source, 'type': 'image', 'caption': ''})
                                cprint('Source image : ' +  source, 'yellow'); print 'Caption :'
                                data.append({'data': text, 'type': 'text'}); cprint(text, 'green')
                    else:
                        data1, thumb = processNode(node); flagError = 'error'
                        if thumbnail!='': thumbnail = thumb; flgImg = True
                        for item in data1: data.append(item)
                else:
                    content     =   pycommonlib.getStringWithNode(node)
                    if flgCaption: 
                        flgCaption = False
                        if content!=None and content!='':
                            if len(content) < 100: 
                                data.append({'data': source_img, 'type': 'image', 'caption': content})
                                cprint('Source image : ' +  source, 'yellow'); print 'Caption :', content
                            else:
                                data.append({'data': source_img, 'type': 'image', 'caption': ''})
                                cprint('Source image : ' +  source, 'yellow'); print 'Caption :'
                                data.append({'data': content, 'type': 'text'}); cprint(content, 'green')
                        else: 
                            data.append({'data': source_img, 'type': 'image', 'caption': ''})
                            cprint('Source image : ' +  source, 'yellow'); print 'Caption :'; 
                        continue 
                    if content!=None or content!='':
                        textBold = False; descendants = node.xpath('.//*')
                        if len(descendants) > 0:
                            for des in descendants: 
                                style = des.get('style')
                                if style and len(content) < 120:
                                    if pycommonlib.regexString('bold', style): textBold = True
                        if len(content) > 1:
                            if textBold: data.append({'data': content, 'type': 'textbold'}); cprint(content, 'yellow')  
                            else: data.append({'data': content, 'type': 'text'}); cprint(content, 'green') 
                    if node.tail:   
                        tailNode = node.tail.strip(); tailNode = ' '.join(tailNode.split())
                        if len(tailNode) > 1: data.append({'data': tailNode, 'type': 'text'}); cprint(tailNode, 'white')
        doc= ({ 'hashUrl'       :   aId,
                'title'         :   title,
                'thumbnail'     :   thumbnail,
                'description'   :   description,
                'content'       :   data,
                'newsLink'      :   link,
                'update'        :   postedDate,
                'source'        :   'dantri.com.vn',
                'category'      :   lft,
                'root'          :   root_id,
                'is_active'     :   True,
                'lastupdate'    :   datetime.datetime.utcnow(),
                'timestamp'     :   time.time(),
                'date'          :   datetime.datetime.utcnow(),
                'tags'          :   tag, })
        if len(data) > 0: ARTICLE_COLLECTION.save(doc)
        print postedDate
        print aId
        cprint('Title :' + title, 'green')
        cprint('Thumbnail : ' + thumbnail, 'green')
        cprint('Intro : ' + description, 'green')
        return 0
    except:
        traceback.print_exc()        
        
def processPage(page, subPath, root_id, lft, tag):
    try:
        gbcount = loop = 0
        lurl    =   '{0}{1}-{2}.htm'.format(SITE_URL, subPath, page)
        cprint('Process page : ' + lurl, 'white')
        tree    =   pycommonlib.getXMLTree(lurl, userAgent=True)
        listNode    =   tree.xpath('//form//table[3]//tr/td/a')
        if len(listNode) < 1: return
        for node in listNode:  
            count = processArticle('{0}{1}'.format(SITE_URL, node.get('href')), root_id, lft, tag)
            if count!=None: gbcount += count; loop += 1
        return gbcount, loop
    except:
        traceback.print_exc(); return None, None
        
def processCategory(cat):
    try:
        gbcount = 0; loop = 0
        cprint('Process category : ' + CATEGORIES[cat]['link'], 'yellow')
        root_id, lft = getRootLftCategory(CATEGORIES[cat]['category'])
        if root_id==None: return
        lurl = '{0}/{1}/{2}.htm'.format(SITE_URL, cat, CATEGORIES[cat]['link'])
        maxPage, subPath = getInfoPage(lurl)
        if subPath=='': return
        for page in range(1, maxPage):
            print 'COUNT, LOOP : (', gbcount, ', ', loop, ')' 
            if gbcount>MAX_COUNT or loop>MAX_ARTICLE:
                cprint('Dừng xử lý do vượt quá số lượng trùng lặp cho phép hoặc vượt quá số lượng tin cần lấy.')
                return 
            c, l = processPage(page, subPath, root_id, lft, CATEGORIES[cat]['tag'])
            if c!=None: gbcount += c; loop += l
    except:
        traceback.print_exc()

def forceQuit():
    try:
        print 'Finished.', datetime.datetime.now()
        pid = os.getpid(); os._exit(1); os.kill(pid, 9)
    except:
        traceback.print_exc()
        
if __name__ == '__main__':
    try:
        cprint('start crawler m.dantri.com.vn', 'yellow')
        '''
        processArticle('http://m.dantri.com.vn/c36/s36-502266/tim-thay-thuoc-no-tai-trang-trai-cua-tay-sung-na-uy.htm', 10, 100, 'test')
        processArticle('http://m.dantri.com.vn/c111/s111-503971/diem-mat-chien-binh-tren-thi-truong-xe-ban-tai.htm', 10, 100, 'test')
        '''
        pool = workerpool.WorkerPool(size=5)
        pool.map(processCategory, CATEGORIES.keys())
        pool.shutdown(); pool.wait()
        forceQuit()
    except:
        traceback.print_exc()
